import fs from 'fs'
import path from 'path'
import superaget from 'superagent'


export interface Analyzer {
  anylyze: (html: string, filePath: string) => string
}

class Crowller {
  private filePath = path.resolve(__dirname, '../../data/course.json')

  async getRawHtml() {
    const result = await superaget.get(this.url)
    return result.text
  }

  writeContent(content: string) {
    fs.writeFileSync(this.filePath, content)
  }

  async initSpiderProcess() {
    const html = await this.getRawHtml()
    const fileContent = this.analyzer.anylyze(html, this.filePath)
    this.writeContent(fileContent)
  }

  constructor(private url: string, private analyzer: Analyzer) {
    this.initSpiderProcess()
  }
}

export default Crowller

